"""
auth：xgt-python
datetime:2021/11/20
爬取小姐姐跳舞视频
"""
# 导入相应的模块
import requests
import parsel
import re


# url  = 'https://www.520mmtv.com/hd/rewu.html'
# headers = {
#     'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36'
# }
# response = requests.get(url=url,headers=headers)
# print(response.text)

# 函数式实现
def get_html(url):
    """
    发送网络请求 直接返回网页源代码
    :param url: url
    :return: html_data
    """
    html_data = requests.get(url).text
    return html_data


# 功能分块更加明确
def parse_data_1(html_data):
    # 解析数据 提取想要的内容 详情页链接
    selector = parsel.Selector(html_data)
    # 数据提取 css re xpath bs4
    # @href: 获取标签的属性内容
    url_list = selector.xpath('//*[@id="index_ajax_list"]/li/div/a/@href').getall()  # 拿到所有的属性内容
    return url_list


# 向目标网站发送请求
html_data = get_html('https://www.520mmtv.com/hd/rewu.html')
# 第一次解析数据 提取详情页链接
url_list = parse_data_1(html_data)
for url in url_list:
    # 向详情页发送请求
    html_data_2 = get_html(url)
    # 解析数据 提取视频播放地址
    # 吧网页源代码不一样的删除 改成(.*?)
    video_url = re.findall('url: "(.*?)",', html_data_2)[0]

    # 保存数据
    # 拿到数据二进制数据
    video_data = requests.get(video_url).content
    # 标题
    title = video_url.split('/')[-1]
    with open(f'video_热舞小姐姐/{title}', mode='wb') as file:
        file.write(video_data)
